/////////////////////////////////////////////////////////////////////////////
// This file is part of the "Java-DAP" project, a Java implementation
// of the OPeNDAP Data Access Protocol.
//
// Copyright (c) 2007 OPeNDAP, Inc.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
/////////////////////////////////////////////////////////////////////////////
using System;
using System.Diagnostics;
using System.IO;
using System.Net;
using System.Text;
using System.Threading;
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using NDODS.DAP.Exceptions;

namespace NDODS.DAP
{
    /// <summary> This class provides support for common OPeNDAP client-side operations such as
    /// dereferencing a OPeNDAP URL, communicating network activity status
    /// to the user and reading local OPeNDAP objects.
    /// <p/>
    /// Unlike its C++ counterpart, this class does not store instances of the DAS,
    /// DDS, etc. objects. Rather, the methods <code>getDAS</code>, etc. return
    /// instances of those objects.
    /// 
    /// </summary>
    /// <author>  jehamby
    /// </author>
    /// <version>  $Revision: 15901 $
    /// </version>
    public class DConnect
    {
        private static readonly log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.Assembly.GetExecutingAssembly().GetName().Name);
        private bool isLogging = true;


        /// <summary> Returns whether a file name or <code>InputStream</code> is being used
        /// instead of a URL.
        /// 
        /// </summary>
        /// <returns> true if a file name or <code>InputStream</code> is being used.
        /// </returns>
        public virtual bool Local
        {
            get { return (fileStream != null); }
        }

        /// <summary> Returns the DAS object from the dataset referenced by this object's URL.
        /// The DAS object is referred to by appending `.das' to the end of a OPeNDAP
        /// URL.
        /// 
        /// </summary>
        /// <returns> the DAS associated with the referenced dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the </throws>
        /// <summary>                               constructor has an error
        /// </summary>
        /// <throws>  IOException           if an error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DAS parser returned an error </throws>
        /// <throws>  DASException          on an error constructing the DAS </throws>
        /// <throws>  DAP2Exception         if an error returned by the remote server </throws>
        public virtual DAS DAS
        {
            get
            {
                Stream is_Renamed;
                if (fileStream != null)
                    is_Renamed = parseMime(fileStream);
                else
                {
                    //UPGRADE_TODO: Class 'java.net.URL' was converted to a 'System.Uri' which does not throw an exception if a URL specifies an unknown protocol. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1132'"
                    Uri url = new Uri(urlString + ".das" + projString + selString);
                    if (dumpDAS)
                    {
                        Trace.TraceInformation("--DConnect.getDAS to " + url);
                        copy(WebRequest.Create(url).GetResponse().GetResponseStream(), Console.OpenStandardOutput());
                        Trace.TraceInformation("\n--DConnect.getDAS END1");
                        DumpBytes(WebRequest.Create(url).GetResponse().GetResponseStream(), 100);
                        Trace.TraceInformation("\n-DConnect.getDAS END2");
                    }
                    is_Renamed = OpenConnection(url);
                }
                DAS das = new DAS();
                try
                {
                    das.parse(is_Renamed);
                }
                finally
                {
                    is_Renamed.Close(); // stream is always closed even if parse() throws exception
                    connection.GetResponse().Close();
                }
                return das;
            }
        }

        private bool dumpStream = false, dumpDAS = false;

        /// <summary> InputStream to use for connection to a file instead of a remote host.</summary>
        private Stream fileStream;

        /// <summary> The last URLConnection used to communicate with the OPeNDAP server.
        /// Note: This is not an HttpURLConnection because file: URL's are allowed.
        /// Theoretically, one could    also run OPeNDAP over FTP, NFS, or any other
        /// protocol, although this is probably not very useful.
        /// </summary>
        private HttpWebRequest connection;

        /// <summary> The current OPeNDAP URL, as a String (will be converted to URL inside of
        /// getDAS(), GetDDS(), and GetData()), without Constraint Expression.
        /// </summary>
        private String urlString;

        /// <summary> The projection portion of the current OPeNDAP CE (including leading "?").</summary>
        private String projString;

        /// <summary> The selection portion of the current OPeNDAP CE (including leading "&").</summary>
        private String selString;

        /// <summary> Whether to accept compressed documents.</summary>
        private bool acceptDeflate;

        /// <summary> The OPeNDAP server version.</summary>
        private ServerVersion ver;


        /// <summary> Returns the <code>ServerVersion</code> of the last connection.</summary>
        /// <returns> the <code>ServerVersion</code> of the last connection.
        /// </returns>
        public virtual ServerVersion ServerVersion
        {
            get { return ver; }
        }

        public bool IsLogging
        {
            get { return isLogging; }
            set { isLogging = value; }
        }

        public virtual void setServerVersion(String sv)
        {
            Console.Out.WriteLine("ServerVersion made with String: " + sv);

            ver = new ServerVersion(sv);
            Console.Out.WriteLine("ServerVersion.getMajor(): " + ver.Major);
            Console.Out.WriteLine("ServerVersion.getMinor(): " + ver.Minor);
        }

        public virtual void setServerVersion(int major, int minor)
        {
            Console.Out.WriteLine("ServerVersion made with int,int: " + major + "," + minor);

            ver = new ServerVersion(major, minor);
            Console.Out.WriteLine("ServerVersion.getMajor(): " + ver.Major);
            Console.Out.WriteLine("ServerVersion.getMinor(): " + ver.Minor);
        }


        /// <summary> Creates an instance bound to url which accepts compressed documents.
        /// 
        /// </summary>
        /// <param name="urlString">connect to this URL.
        /// </param>
        /// <throws>  FileNotFoundException thrown if <code>urlString</code> is not </throws>
        /// <summary>                               a valid URL, or a filename which exists on the system.
        /// </summary>
        /// <seealso cref="DConnect">
        /// </seealso>
        public DConnect(String urlString) : this(urlString, true)
        {
        }

        /// <summary> Creates an instance bound to url. If <code>acceptDeflate</code> is true
        /// then HTTP Request headers will indicate to servers that this client can
        /// accept compressed documents.
        /// 
        /// </summary>
        /// <param name="urlString">    Connect to this URL.  If urlString is not a valid URL,
        /// it is assumed to be a filename, which is opened.
        /// </param>
        /// <param name="acceptDeflate">true if this client can accept responses encoded
        /// with deflate.
        /// </param>
        /// <throws>  FileNotFoundException thrown if <code>urlString</code> is not </throws>
        /// <summary>                               a valid URL, or a filename which exists on the system.
        /// </summary>
        public DConnect(String urlString, bool acceptDeflate)
        {
            int ceIndex = urlString.IndexOf('?');
            if (ceIndex != - 1)
            {
                this.urlString = urlString.Substring(0, (ceIndex) - (0));
                String expr = urlString.Substring(ceIndex);
                int selIndex = expr.IndexOf('&');
                if (selIndex != - 1)
                {
                    this.projString = expr.Substring(0, (selIndex) - (0));
                    this.selString = expr.Substring(selIndex);
                }
                else
                {
                    this.projString = expr;
                    this.selString = "";
                }
            }
            else
            {
                this.urlString = urlString;
                this.projString = this.selString = "";
            }
            this.acceptDeflate = acceptDeflate;

            // Test if the URL is really a filename, and if so, open the file
            try
            {
                //UPGRADE_TODO: Class 'java.net.URL' was converted to a 'System.Uri' which does not throw an exception if a URL specifies an unknown protocol. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1132'"
                Uri testURL = new Uri(urlString);
            }
            catch (UriFormatException)
            {
                //UPGRADE_TODO: Constructor 'java.io.FileInputStream.FileInputStream' was converted to 'System.IO.FileStream.FileStream' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javaioFileInputStreamFileInputStream_javalangString'"
                fileStream = new FileStream(urlString, FileMode.Open, FileAccess.Read);
            }
        }

        /// <summary> Creates an instance bound to an already open <code>InputStream</code>.
        /// 
        /// </summary>
        /// <param name="is_Renamed">the <code>InputStream</code> to open.
        /// </param>
        public DConnect(Stream is_Renamed)
        {
            this.fileStream = is_Renamed;
        }

        /// <summary> Returns the constraint expression supplied with the URL given to the
        /// constructor. If no CE was given this returns an empty <code>String</code>.
        /// <p/>
        /// Note that the CE supplied to one of this object's constructors is
        /// "sticky"; it will be used with every data request made with this object.
        /// The CE passed to <code>GetData</code>, however, is not sticky; it is used
        /// only for that specific request. This method returns the sticky CE.
        /// 
        /// </summary>
        /// <returns> the constraint expression associated with this connection.
        /// </returns>
        public String CE()
        {
            return projString + selString;
        }

        /// <summary> Returns the URL supplied to the constructor. If the URL contained a
        /// constraint expression that is not returned.
        /// 
        /// </summary>
        /// <returns> the URL of this connection.
        /// </returns>
        public String URL()
        {
            return urlString;
        }

        /// <summary> Open a connection to the OPeNDAP server.
        /// 
        /// </summary>
        /// <param name="url">the URL to open.
        /// </param>
        /// <returns> the opened <code>InputStream</code>.
        /// </returns>
        /// <throws>  IOException   if an IO exception occurred. </throws>
        /// <throws>  DAP2Exception if the OPeNDAP server returned an error. </throws>
        private Stream OpenConnection(Uri url)
        {
            WebClient webClientConnection = new WebClient();
            
            
            //connection = (HttpWebRequest) WebRequest.Create(url);
            if (acceptDeflate)
                webClientConnection.Headers.Set("Accept-Encoding", "deflate");
               // SupportClass.URLConnectionSupport.SetRequestProperty(connection, "Accept-Encoding", "deflate");


            Stream is_Renamed;
            try
            {
                is_Renamed = webClientConnection.OpenRead(url);
                //is_Renamed = connection.GetResponse().GetResponseStream(); // get the HTTP InputStream
                //SupportClass.DumpStreamToDisk(is_Renamed, "");
            }
            catch (NullReferenceException)
            {
                throw new DAP2Exception("Connection cannot be opened; connection is null");
            }
            catch (FileNotFoundException)
            {
                throw new DAP2Exception("Connection cannot be opened; file not found");
            }
            if (is_Renamed == null)
                throw new DAP2Exception("Unable to open input stream");

            // check headers
            //String type = connection.GetResponse().Headers.Get("content-description");
            String type = webClientConnection.ResponseHeaders.Get("content-description");

            HandleContentDesc(is_Renamed, type);

            //UPGRADE_TODO: Method 'java.net.URLConnection.getHeaderField' was converted to 'System.Net.WebRequest.GetResponse.Headers.Get' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javanetURLConnectiongetHeaderField_javalangString'"
            //String sHeader_server = connection.GetResponse().Headers.Get("XDODS-Server");
            String sHeader_server = webClientConnection.ResponseHeaders.Get("XDODS-Server");
            if (sHeader_server == null)
                throw new DAP2Exception("Not a valid OPeNDAP server - " + "Missing MIME Header field \"xdods-server.\"");
            ver = new ServerVersion(sHeader_server);

            //System.err.println("Server: " + ver + ": " +
            //                    ver.getMajor() + "," + ver.getMinor());

            //UPGRADE_TODO: Method 'java.net.URLConnection.getContentEncoding' was converted to 'SupportClass.URLConnectionSupport.GetContentEncoding' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javanetURLConnectiongetContentEncoding'"
            //String encoding = SupportClass.URLConnectionSupport.GetContentEncoding(connection);
            String encoding = webClientConnection.ResponseHeaders.Get("Content-Encoding");

            //System.err.println("Content Encoding: " + encoding);
            return HandleContentEncoding(is_Renamed, encoding);
        }

        /// <summary> Returns the DDS object from the dataset referenced by this object's URL.
        /// The DDS object is referred to by appending `.dds' to the end of a OPeNDAP
        /// URL.
        /// 
        /// </summary>
        /// <returns> the DDS associated with the referenced dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if an error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if an error returned by the remote server </throws>
        public virtual DDS GetDDS()
        {
            return (GetDDS(""));
        }


        /// <summary> Use some sense when assembling the CE. Since this DConnect
        /// object may have constructed using a CE, any new CE will
        /// have to be integrated into it for subsequent requests.
        /// Try to do this in a sensible manner!
        /// 
        /// </summary>
        /// <param name="CE">The new CE from the client.
        /// </param>
        /// <returns> The complete CE (the one this object was built
        /// with integrated with the clients)
        /// </returns>
        private String getCompleteCE(String CE)
        {
            String localProjString, localSelString;
            int selIndex = CE.IndexOf('&');
            if (selIndex != - 1)
            {
                if (CE.IndexOf('?') == 0)
                    localProjString = CE.Substring(1, (selIndex) - (1));
                else
                    localProjString = CE.Substring(0, (selIndex) - (0));

                localSelString = CE.Substring(selIndex);
            }
            else
            {
                if (CE.IndexOf('?') == 0)
                    localProjString = CE.Substring(1);
                else
                    localProjString = CE;
                localSelString = "";
            }

            String ce = projString;

            if (!localProjString.Equals(""))
            {
                if (!ce.Equals("") && localProjString.IndexOf(',') != 0)
                    ce += ",";
                ce += localProjString;
            }

            if (!selString.Equals(""))
            {
                if (selString.IndexOf('&') != 0)
                    ce += "&";
                ce += selString;
            }

            if (!localSelString.Equals(""))
            {
                if (localSelString.IndexOf('&') != 0)
                    ce += "&";
                ce += localSelString;
            }

            if (ce.IndexOf('?') != 0)
            {
                ce = "?" + ce;
            }

            return (ce);
        }

        /// <summary> Returns the DDS object from the dataset referenced by this object's URL.
        /// The DDS object is referred to by appending `.dds' to the end of a OPeNDAP
        /// URL.
        /// 
        /// </summary>
        /// <returns> the DDS associated with the referenced dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if an error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if an error returned by the remote server </throws>
        public virtual DDS GetDDS(String CE)
        {
            Stream is_Renamed;
            if (fileStream != null)
                is_Renamed = parseMime(fileStream);
            else
            {
                //UPGRADE_TODO: Class 'java.net.URL' was converted to a 'System.Uri' which does not throw an exception if a URL specifies an unknown protocol. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1132'"
                Uri url = new Uri(urlString + ".dds" + getCompleteCE(CE));

                is_Renamed = OpenConnection(url);
                //System.out.println("Opened DDS URL: "+url);
            }
            DDS dds = new DDS();
            try
            {
                dds.parse(new BinaryReader(is_Renamed));
            }
            finally
            {
                is_Renamed.Close(); // stream is always closed even if parse() throws exception
                connection.GetResponse().Close();
            }
            return dds;
        }

        /// <summary> Returns the DDS object from the dataset referenced by this object's URL.
        /// The DDS object is referred to by appending `.ddx' to the end of a OPeNDAP
        /// URL. The server should send back a DDX (A DDS in XML format) which
        /// will get parsed here (locally) and a new DDS instantiated using
        /// the DDSXMLParser.
        /// 
        /// </summary>
        /// <returns> the DDS associated with the referenced dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if an error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if an error returned by the remote server </throws>
        /// <opendap.ddx.experimental>  </opendap.ddx.experimental>
        public virtual DDS GetDDX()
        {
            return (GetDDX(""));
        }


        /// <summary> Returns the DDS object from the dataset referenced by this object's URL.
        /// The DDS object is referred to by appending `.ddx' to the end of a OPeNDAP
        /// URL. The server should send back a DDX (A DDS in XML format) which
        /// will get parsed here (locally) and a new DDS instantiated using
        /// the DDSXMLParser.
        /// 
        /// </summary>
        /// <param name="CE">
        /// </param>
        /// <returns> the DDS associated with the referenced dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  MalformedURLException </throws>
        /// <throws>  IOException           if an error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if an error returned by the remote server </throws>
        /// <opendap.ddx.experimental>  </opendap.ddx.experimental>
        public virtual DDS GetDDX(String CE)
        {
            Stream is_Renamed;

            //UPGRADE_TODO: Class 'java.net.URL' was converted to a 'System.Uri' which does not throw an exception if a URL specifies an unknown protocol. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1132'"
            Uri url = new Uri(urlString + ".ddx" + getCompleteCE(CE));

            if (fileStream != null)
                is_Renamed = parseMime(fileStream);
            else
            {
                is_Renamed = OpenConnection(url);
                Console.Out.WriteLine("Opened DDX URL: " + url);
            }

            DDS dds = new DDS();
            try
            {
                dds.parseXML(is_Renamed, false);
            }
            finally
            {
                is_Renamed.Close(); // stream is always closed even if parse() throws exception
                connection.GetResponse().Close();
            }
            return dds;
        }


        /// <summary> Returns the DataDDS object from the dataset referenced by this object's URL.
        /// The DDS object is referred to by appending `.ddx' to the end of a OPeNDAP
        /// URL. The server should send back a DDX (A DDS in XML format) which
        /// will get parsed here (locally) and a new DDS instantiated using
        /// the DDSXMLParser.
        /// 
        /// </summary>
        /// <returns> the DataDDS associated with the referenced dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if an error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if an error returned by the remote server </throws>
        /// <opendap.ddx.experimental>  </opendap.ddx.experimental>
        public virtual DataDDS GetDataDDX()
        {
            return (GetDataDDX("", new DefaultFactory()));
        }


        /// <summary> Returns the DataDDS object from the dataset referenced by this object's URL.
        /// The DDS object is referred to by appending `.ddx' to the end of a OPeNDAP
        /// URL. The server should send back a DDX (A DDS in XML format) which
        /// will get parsed here (locally) and a new DDS instantiated using
        /// the DDSXMLParser.
        /// 
        /// </summary>
        /// <param name="CE">The constraint expression to use for this request.
        /// </param>
        /// <returns> the DataDDS associated with the referenced dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if an error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if an error returned by the remote server </throws>
        /// <opendap.ddx.experimental>  </opendap.ddx.experimental>
        public virtual DataDDS GetDataDDX(String CE)
        {
            return (GetDataDDX(CE, new DefaultFactory()));
        }

        /// <summary> Returns the DataDDS object from the dataset referenced by this object's URL.
        /// The DDS object is referred to by appending `.ddx' to the end of a OPeNDAP
        /// URL. The server should send back a DDX (A DDS in XML format) which
        /// will get parsed here (locally) and a new DDS instantiated using
        /// the DDSXMLParser.
        /// 
        /// </summary>
        /// <param name="CE"> The constraint expression to use for this request.
        /// </param>
        /// <param name="btf">The <code>BaseTypeFactory</code> to build the member
        /// variables in the DDS with.
        /// </param>
        /// <returns> the DataDDS associated with the referenced dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if an error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if an error returned by the remote server </throws>
        /// <seealso cref="IBaseTypeFactory">
        /// </seealso>
        /// <opendap.ddx.experimental>  </opendap.ddx.experimental>
        public virtual DataDDS GetDataDDX(String CE, IBaseTypeFactory btf)
        {
            Stream is_Renamed;

            //UPGRADE_TODO: Class 'java.net.URL' was converted to a 'System.Uri' which does not throw an exception if a URL specifies an unknown protocol. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1132'"
            Uri url = new Uri(urlString + ".ddx" + getCompleteCE(CE));

            if (fileStream != null)
                is_Renamed = parseMime(fileStream);
            else
            {
                is_Renamed = OpenConnection(url);

                Console.Out.WriteLine("Opened DataDDX URL: " + url);
            }

            DataDDS dds = new DataDDS(ver, btf);
            try
            {
                dds.parseXML(is_Renamed, false);
            }
            finally
            {
                is_Renamed.Close(); // stream is always closed even if parse() throws exception
                connection.GetResponse().Close();
            }
            return dds;
        }


        /// <summary> Returns the `Data object' from the dataset referenced by this object's
        /// URL given the constraint expression CE. Note that the Data object is
        /// really just a DDS object with data bound to the variables. The DDS will
        /// probably contain fewer variables (and those might have different
        /// types) than in the DDS returned by GetDDS() because that method returns
        /// the entire DDS (but without any data) while this method returns
        /// only those variables listed in the projection part of the constraint
        /// expression.
        /// <p/>
        /// Note that if CE is an empty String then the entire dataset will be
        /// returned, unless a "sticky" CE has been specified in the constructor.
        /// 
        /// </summary>
        /// <param name="CE">      The constraint expression to be applied to this request by the
        /// server.  This is combined with any CE given in the constructor.
        /// </param>
        /// <param name="statusUI">the <code>StatusUI</code> object to use for GUI updates
        /// and user cancellation notification (may be null).
        /// </param>
        /// <param name="btf"></param>
        /// <returns> The <code>DataDDS</code> object that results from applying the
        /// given CE, combined with this object's sticky CE, on the referenced
        /// dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if any error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if any error returned by the remote server </throws>
        public virtual DataDDS GetData(String CE, IStatusUI statusUI, IBaseTypeFactory btf)
        {
            if (fileStream != null)
                return GetDataFromFileStream(fileStream, statusUI, btf);

            //UPGRADE_TODO: Class 'java.net.URL' was converted to a 'System.Uri' which does not throw an exception if a URL specifies an unknown protocol. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1132'"
            Uri url = new Uri(urlString + ".dods" + getCompleteCE(CE));

            String errorMsg = "DConnect GetData failed. URL: " + url;
            int errorCode = DAP2Exception.UNKNOWN_ERROR;
            try
            {
                return GetDataFromUrl(url, statusUI, btf);
            }
            catch (DAP2Exception e)
            {
                errorMsg = e.ErrorMessage;
                errorCode = e.ErrorCode;
                return null;
            }
        }

        /// <summary> Returns the `Data object' from the dataset referenced by this object's
        /// URL given the constraint expression CE. Note that the Data object is
        /// really just a DDS object with data bound to the variables. The DDS will
        /// probably contain fewer variables (and those might have different
        /// types) than in the DDS returned by GetDDS() because that method returns
        /// the entire DDS (but without any data) while this method returns
        /// only those variables listed in the projection part of the constraint
        /// expression.
        /// <p/>
        /// Note that if CE is an empty String then the entire dataset will be
        /// returned, unless a "sticky" CE has been specified in the constructor.
        /// <p/>
        /// <p/>
        /// This method uses the 2 step method for aquiring data from a server using
        /// a DDX and a BLOB. First, a DDX (an XML representation of a DDS) is requested.
        /// The DDX is parsed and a DataDDS is created.
        /// The DDX contains a URL that points to the servers BLOB service. The BLOB
        /// service returns only the serialized binary content of the DataDDS. The DataDDS
        /// then deserializes the BLOB and fills itself with data.
        /// 
        /// </summary>
        /// <param name="CE">      The constraint expression to be applied to this request by the
        /// server.  This is combined with any CE given in the constructor.
        /// </param>
        /// <param name="statusUI">the <code>StatusUI</code> object to use for GUI updates
        /// and user cancellation notification (may be null).
        /// </param>
        /// <param name="btf"></param>
        /// <returns> The <code>DataDDS</code> object that results from applying the
        /// given CE, combined with this object's sticky CE, on the referenced
        /// dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if any error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if any error returned by the remote server </throws>
        /// <opendap.ddx.experimental>  </opendap.ddx.experimental>
        public virtual DataDDS GetDDXData(String CE, IStatusUI statusUI, IBaseTypeFactory btf)
        {
            if (fileStream != null)
                throw new UriFormatException("Cannont Read DDX data from a file. " +
                                             "The DDX client/server interaction " +
                                             "requires 2 request/response pairs. " +
                                             "File based input is not currently supported.");


            //UPGRADE_TODO: Class 'java.net.URL' was converted to a 'System.Uri' which does not throw an exception if a URL specifies an unknown protocol. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1132'"
            Uri url = new Uri(urlString + ".ddx" + getCompleteCE(CE));


            String errorMsg = "DConnect GetDDXData failed. URL: " + url.AbsoluteUri;
            int errorCode = DAP2Exception.UNKNOWN_ERROR;
            int retry = 1;
            long backoff = 100L;
            while (true)
            {
                try
                {
                    return GetDDXDataFromURL(url, statusUI, btf);
                }
                catch (DAP2Exception e)
                {
                    Console.Out.WriteLine("DConnect GetData failed; retry (" + retry + "," + backoff + ") " + url);
                    errorMsg = e.ErrorMessage;
                    errorCode = e.ErrorCode;

                    try
                    {
                        SupportClass.ThreadClass.Current();
                        //UPGRADE_TODO: Method 'java.lang.Thread.sleep' was converted to 'System.Threading.Thread.Sleep' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javalangThreadsleep_long'"
                        Thread.Sleep(new TimeSpan(10000*backoff));
                    }
                    catch (ThreadInterruptedException)
                    {
                    }
                }

                if (retry == 5)
                    throw new DAP2Exception(errorCode, errorMsg);
                retry++;
                backoff *= 2;
            }
        }


        private DataDDS GetDataFromFileStream(Stream pFileStream, IStatusUI statusUI, IBaseTypeFactory btf)
        {
            Stream is_Renamed = parseMime(pFileStream);
            DataDDS dds = new DataDDS(ver, btf);

            try
            {
                dds.parse(new HeaderInputStream(is_Renamed)); // Read the DDS header
                // NOTE: the HeaderInputStream will have skipped over "Data:" line
                dds.ReadData(is_Renamed, statusUI); // Read the data!
            }
            finally
            {
                is_Renamed.Close(); // stream is always closed even if parse() throws exception
            }
            return dds;
        }


        /// <summary> Opens the BLOB uRL in the DDS supplied and deserializes that binary content
        /// sent from the server cooresponding to the DDS.
        /// <p/>
        /// <p/>
        /// This method is the 2nd step in the 2 step process for aquiring data from a server using
        /// a DDX and a BLOB. First, a DDX (an XML representation of a DDS) is requested.
        /// The DDX is parsed and a DataDDS is created.
        /// The DDX contains a URL that points to the servers BLOB service. The BLOB
        /// service returns only the serialized binary content of the DataDDS. The DataDDS
        /// then deserializes the BLOB and fills itself with data.
        /// 
        /// </summary>
        /// <param name="dds">     The DDS containing the BLOB URL and into which the BLOB
        /// (serialized binary content) will be deserialized.
        /// </param>
        /// <param name="statusUI">the <code>StatusUI</code> object to use for GUI updates
        /// and user cancellation notification (may be null).
        /// </param>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if any error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if any error returned by the remote server </throws>
        public virtual void getBlobData(DataDDS dds, IStatusUI statusUI)
        {
            bool dumpStreamErr = false; // opendap.util.Debug.isSet("dumpStreamErr");

            Console.Out.WriteLine("dds.getBlobURL(): " + dds.BlobContentID);

            if (dds.BlobContentID == null)
            {
                throw new UriFormatException("Blob URL was 'null'. " +
                                             "This may indicate that this OPeNDAP Server does not support the full use of DDX.");
            }


            //UPGRADE_TODO: Class 'java.net.URL' was converted to a 'System.Uri' which does not throw an exception if a URL specifies an unknown protocol. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1132'"
            Uri blobURL = new Uri(dds.BlobContentID);

            Console.Out.WriteLine("Opening BLOB URL: " + blobURL);

            Stream is_Renamed = OpenConnection(blobURL);

            // - - - - -  DEBUG - - - - - -
            MemoryStream bis = null;
            if (dumpStream)
            {
                Console.Out.WriteLine("DConnect to " + blobURL);
                MemoryStream bos = new MemoryStream();
                copy(is_Renamed, bos);
                bis = new MemoryStream(SupportClass.ToByteArray(SupportClass.ToSByteArray(bos.ToArray())));
                is_Renamed = bis;
            }
            if (dumpStreamErr)
            {
                MemoryStream bos = new MemoryStream();
                copy(is_Renamed, bos);
                bis = new MemoryStream(SupportClass.ToByteArray(SupportClass.ToSByteArray(bos.ToArray())));
                is_Renamed = bis;
            }
            // - - - - - - - - - - - - - - -

            try
            {
                dds.ReadData(is_Renamed, statusUI); // Read the data!
            }
                //UPGRADE_NOTE: Exception 'java.lang.Throwable' was converted to 'System.Exception' which has different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1100'"
            catch (Exception e)
            {
                Console.Out.WriteLine("DConnect dds.ReadData problem with: " + blobURL + "\nStack Trace:");
                SupportClass.WriteStackTrace(e, Console.Out);

                // - - - - -  DEBUG - - - - - -
                if (dumpStreamErr)
                {
                    Console.Out.WriteLine("DConnect dump " + blobURL);
                    bis.Position = SupportClass.ByteArrayInputManager.manager.ResetMark(bis);
                    dump(bis);
                }
                // - - - - - - - - - - - - - - -

                //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
                throw new DAP2Exception("Connection problem when reading: " + blobURL + "\n" + "Error Message - " +
                                        e.Message);
            }
            finally
            {
                is_Renamed.Close(); // stream is always closed even if parse() throws exception
                connection.GetResponse().Close();
            }
        }


        /// <summary> Returns the `Data object' from the dataset referenced by this object's
        /// URL given the constraint expression CE. Note that the Data object is
        /// really just a DDS object with data bound to the variables. The DDS will
        /// probably contain fewer variables (and those might have different
        /// types) than in the DDS returned by GetDDS() because that method returns
        /// the entire DDS (but without any data) while this method returns
        /// only those variables listed in the projection part of the constraint
        /// expression.
        /// <p/>
        /// Note that if CE is an empty String then the entire dataset will be
        /// returned, unless a "sticky" CE has been specified in the constructor.
        /// <p/>
        /// <p/>
        /// This method uses the 2 step method for aquiring data from a server using
        /// a DDX and a BLOB. First, a DDX (an XML representation of a DDS) is requested.
        /// The DDX is parsed and a DataDDS is created.
        /// The DDX contains a URL that points to the servers BLOB service. The BLOB
        /// service returns only the serialized binary content of the DataDDS. The DataDDS
        /// then deserializes the BLOB and fills itself with data.
        /// 
        /// </summary>
        /// <param name="url">     The complete URL of the dataset. Constraint Expression included.
        /// </param>
        /// <param name="statusUI">the <code>StatusUI</code> object to use for GUI updates
        /// and user cancellation notification (may be null).
        /// </param>
        /// <param name="btf">     The <code>BaseTypeFactory</code> to build the member
        /// variables in the DDS with.
        /// </param>
        /// <returns> The <code>DataDDS</code> object that results from applying the
        /// given CE, combined with this object's sticky CE, on the referenced
        /// dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if any error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if any error returned by the remote server </throws>
        /// <opendap.ddx.experimental>  </opendap.ddx.experimental>
        public virtual DataDDS GetDDXDataFromURL(Uri url, IStatusUI statusUI, IBaseTypeFactory btf)
        {
            Console.Out.WriteLine("Opening DDX URL: " + url);
            Stream is_Renamed = OpenConnection(url);
            DataDDS dds = new DataDDS(ver, btf);

            bool dumpStreamErr = false; // opendap.util.Debug.isSet("dumpStreamErr");

            // - - - - -  DEBUG - - - - - -
            MemoryStream bis = null;
            if (dumpStream)
            {
                Console.Out.WriteLine("DConnect to " + url);
                MemoryStream bos = new MemoryStream();
                copy(is_Renamed, bos);
                bis = new MemoryStream(SupportClass.ToByteArray(SupportClass.ToSByteArray(bos.ToArray())));
                is_Renamed = bis;
            }
            if (dumpStreamErr)
            {
                MemoryStream bos = new MemoryStream();
                copy(is_Renamed, bos);
                bis = new MemoryStream(SupportClass.ToByteArray(SupportClass.ToSByteArray(bos.ToArray())));
                is_Renamed = bis;
            }
            // - - - - - - - - - - - - - - -

            try
            {
                // - - - - -  DEBUG - - - - - -
                if (dumpStream)
                {
                    SupportClass.ByteArrayInputManager.manager.MarkPosition(1000, bis);
                    Console.Out.WriteLine("DConnect parse header: ");
                    dump(bis);
                    bis.Position = SupportClass.ByteArrayInputManager.manager.ResetMark(bis);
                }
                // - - - - - - - - - - - - - - -

                dds.parseXML(is_Renamed, false); // Read the DDX

                //dds.parse(new HeaderInputStream(is));    // Read the DDS header
                // NOTE: the HeaderInputStream will have skipped over "Data:" line

                // - - - - -  DEBUG - - - - - -
                if (dumpStream)
                {
                    SupportClass.ByteArrayInputManager.manager.MarkPosition(20, bis);
                    Console.Out.WriteLine("DConnect done with header, next bytes are: ");
                    DumpBytes(bis, 20);
                    bis.Position = SupportClass.ByteArrayInputManager.manager.ResetMark(bis);
                }
                // - - - - - - - - - - - - - - -
            }
                //UPGRADE_NOTE: Exception 'java.lang.Throwable' was converted to 'System.Exception' which has different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1100'"
            catch (Exception e)
            {
                Console.Out.WriteLine("DConnect ddx.parse problem with: " + url + "\nStack Trace:");
                SupportClass.WriteStackTrace(e, Console.Out);

                // - - - - -  DEBUG - - - - - -
                if (dumpStreamErr)
                {
                    Console.Out.WriteLine("DConnect dump " + url);
                    bis.Position = SupportClass.ByteArrayInputManager.manager.ResetMark(bis);
                    dump(bis);
                }
                // - - - - - - - - - - - - - - -

                //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
                throw new DAP2Exception("Connection problem when reading: " + url + "\n" + "Error Message - " +
                                        e.Message);
            }
            finally
            {
                is_Renamed.Close(); // stream is always closed even if parse() throws exception
                connection.GetResponse().Close();
            }


            getBlobData(dds, statusUI);


            return dds;
        }


        public virtual DataDDS GetDataFromUrl(Uri url, IStatusUI statusUI, IBaseTypeFactory btf)
        {
            Stream is_Renamed = OpenConnection(url);
            DataDDS dds = new DataDDS(ver, btf);

            // DEBUG
            MemoryStream bis = null;
            if (IsLogging)
            {
                log.Info("DConnect to " + url);
                bis = new MemoryStream();
                SupportClass.CopyStream(is_Renamed, bis);
            }

            String operation = "";
            try
            {
                if (IsLogging)
                {
                    log.Info("DConnect parse header: ");
                    DumpDODSHeader(bis);
                }

                operation = "dds.parse";
                dds.parse(new HeaderInputStream(is_Renamed)); // Read the DDS header
                // NOTE: the HeaderInputStream will have skipped over "Data:" line

                if (IsLogging)
                {
                    log.Info("DConnect done with header, next bytes are: ");
                    DumpBytes(bis, 20);
                }

                operation = "dds.ReadData";
                dds.ReadData(is_Renamed, statusUI); // Read the data!
            }
                //UPGRADE_NOTE: Exception 'java.lang.Throwable' was converted to 'System.Exception' which has different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1100'"
            catch (Exception e)
            {
                //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
                log.Error("DConnect dds.parse: " + url + "\n " + e);
                log.Error(e.StackTrace);

                //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
                throw new DAP2Exception("Connection problem when reading: " + url + "\n" + "Error Message - " +
                                        e.Message);
            }
            finally
            {
                is_Renamed.Close(); // stream is always closed even if parse() throws exception
                //connection.GetResponse().Close();
            }

            return dds;
        }



        // DEBUG JC
        private void DumpDODSHeader(Stream is_Renamed)
        {
            //UPGRADE_TODO: Class 'java.io.DataInputStream' was converted to 'System.IO.BinaryReader' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javaioDataInputStream'"
            BinaryReader d = new BinaryReader(is_Renamed);
            try
            {
                long available;

                available = is_Renamed.Length - is_Renamed.Position;
                log.Info("dump lines available=" + (int)available);

                long currentPosition = 0;
                while (true)
                {
                    //UPGRADE_TODO: Method 'java.io.DataInputStream.readLine' was converted to 'System.IO.BinaryReader.ReadString' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javaioDataInputStreamreadLine'"
                    StringBuilder sb = new StringBuilder(100);

                    char currentChar = d.ReadChar();
                    currentPosition++;
                    while (currentChar.ToString() != "\n" && (d.BaseStream.Position < d.BaseStream.Length))
                    {
                        sb.Append(currentChar);
                        currentChar = d.ReadChar();
                        currentPosition++;
                    }
                    String line = sb.ToString();

                    log.Info(line);

                    if (null == line)
                        return;
                    if (line.Equals("Data:"))
                        break;
                }
                long available2;
                available2 = is_Renamed.Length - is_Renamed.Position;
                log.Info("dump bytes available=" + (int)available2);
                DumpBytes(is_Renamed, 20);
            }
            catch (Exception) { }
        }

        private void DumpBytes(Stream is_Renamed, int n)
        {
            try
            {
                //UPGRADE_TODO: Class 'java.io.DataInputStream' was converted to 'System.IO.BinaryReader' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javaioDataInputStream'"
                BinaryReader d = new BinaryReader(is_Renamed);
                int count = 0;
                long available;
                available = d.BaseStream.Length - d.BaseStream.Position;
                while ((count < n) && ((int)available > 0))
                {
                    log.Info(count + " " + (sbyte)d.ReadByte());
                    count++;
                    available = d.BaseStream.Length - d.BaseStream.Position;
                }
            }
            catch (Exception) { }
        }

        // DEBUG JC
        private void copy(Stream in_Renamed, Stream out_Renamed)
        {
            try
            {
                sbyte[] buffer = new sbyte[256];
                while (true)
                {
                    int bytesRead = SupportClass.ReadInput(in_Renamed, buffer, 0, buffer.Length);
                    if (bytesRead == - 1)
                        break;
                    out_Renamed.Write(SupportClass.ToByteArray(buffer), 0, bytesRead);
                }
            }
            catch (IOException )
            {
                // Don't bother to do a thing
            }
        }


        // DEBUG JC
        private void dump(Stream is_Renamed)
        {
            //UPGRADE_TODO: Class 'java.io.DataInputStream' was converted to 'System.IO.BinaryReader' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javaioDataInputStream'"
            BinaryReader d = new BinaryReader(is_Renamed);

            try
            {
                long available;
                available = is_Renamed.Length - is_Renamed.Position;
                Console.Out.WriteLine("dump lines avail=" + (int) available);
                while (true)
                {
                    //UPGRADE_TODO: Method 'java.io.DataInputStream.readLine' was converted to 'System.IO.BinaryReader.ReadString' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javaioDataInputStreamreadLine'"
                    String line = d.ReadString();
                    Console.Out.WriteLine(line);
                    if (null == line)
                        return;
                    if (line.Equals("Data:"))
                        break;
                }
                long available2;
                available2 = is_Renamed.Length - is_Renamed.Position;
                Console.Out.WriteLine("dump bytes avail=" + (int) available2);
                DumpBytes(is_Renamed, 20);
            }
            catch (EndOfStreamException)
            {
                // Don't bother to do a thing
            }
        }



        /// <summary> Returns the `Data object' from the dataset referenced by this object's
        /// URL given the constraint expression CE. Note that the Data object is
        /// really just a DDS object with data bound to the variables. The DDS will
        /// probably contain fewer variables (and those might have different
        /// types) than in the DDS returned by GetDDS() because that method returns
        /// the entire DDS (but without any data) while this method returns
        /// only those variables listed in the projection part of the constraint
        /// expression.
        /// <p/>
        /// Note that if CE is an empty String then the entire dataset will be
        /// returned, unless a "sticky" CE has been specified in the constructor.
        /// 
        /// </summary>
        /// <param name="CE">      The constraint expression to be applied to this request by the
        /// server.  This is combined with any CE given in the constructor.
        /// </param>
        /// <param name="statusUI">the <code>StatusUI</code> object to use for GUI updates
        /// and user cancellation notification (may be null).
        /// </param>
        /// <returns> The <code>DataDDS</code> object that results from applying the
        /// given CE, combined with this object's sticky CE, on the referenced
        /// dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if any error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if any error returned by the remote server </throws>
        public virtual DataDDS GetData(String CE, IStatusUI statusUI)
        {
            return GetData(CE, statusUI, new DefaultFactory());
        }


        /// <summary> Returns the `Data object' from the dataset referenced by this object's
        /// URL given the constraint expression CE. Note that the Data object is
        /// really just a DDS object with data bound to the variables. The DDS will
        /// probably contain fewer variables (and those might have different
        /// types) than in the DDS returned by GetDDS() because that method returns
        /// the entire DDS (but without any data) while this method returns
        /// only those variables listed in the projection part of the constraint
        /// expression.
        /// <p/>
        /// Note that if CE is an empty String then the entire dataset will be
        /// returned, unless a "sticky" CE has been specified in the constructor.
        /// <p/>
        /// <p/>
        /// This method uses the 2 step method for aquiring data from a server using
        /// a DDX and a BLOB. First, a DDX (an XML representation of a DDS) is requested.
        /// The DDX is parsed and a DataDDS is created.
        /// The DDX contains a URL that points to the servers BLOB service. The BLOB
        /// service returns only the serialized binary content of the DataDDS. The DataDDS
        /// then deserializes the BLOB and fills itself with data.
        /// 
        /// </summary>
        /// <param name="CE">      The constraint expression to be applied to this request by the
        /// server.  This is combined with any CE given in the constructor.
        /// </param>
        /// <param name="statusUI">the <code>StatusUI</code> object to use for GUI updates
        /// and user cancellation notification (may be null).
        /// </param>
        /// <returns> The <code>DataDDS</code> object that results from applying the
        /// given CE, combined with this object's sticky CE, on the referenced
        /// dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if any error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if any error returned by the remote server </throws>
        /// <opendap.ddx.experimental>  </opendap.ddx.experimental>
        public virtual DataDDS GetDDXData(String CE, IStatusUI statusUI)
        {
            return GetDDXData(CE, statusUI, new DefaultFactory());
        }


        /// <summary> Return the data object with no local constraint expression.  Same as
        /// <code>GetData("", statusUI)</code>.
        /// 
        /// </summary>
        /// <param name="statusUI">the <code>StatusUI</code> object to use for GUI updates
        /// and user cancellation notification (may be null).
        /// </param>
        /// <returns> The <code>DataDDS</code> object that results from applying
        /// this object's sticky CE, if any, on the referenced dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if any error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if any error returned by the remote server </throws>
        /// <seealso cref="DConnect.GetData(String, IStatusUI, IBaseTypeFactory)">
        /// </seealso>
        public DataDDS GetData(IStatusUI statusUI)
        {
            return GetData("", statusUI, new DefaultFactory());
        }

        /// <summary> Returns the `Data object' from the dataset referenced by this object's
        /// URL given the constraint expression CE. Note that the Data object is
        /// really just a DDS object with data bound to the variables. The DDS will
        /// probably contain fewer variables (and those might have different
        /// types) than in the DDS returned by GetDDS() because that method returns
        /// the entire DDS (but without any data) while this method returns
        /// only those variables listed in the projection part of the constraint
        /// expression.
        /// <p/>
        /// Note that if CE is an empty String then the entire dataset will be
        /// returned, unless a "sticky" CE has been specified in the constructor.
        /// <p/>
        /// <p/>
        /// This method uses the 2 step method for aquiring data from a server using
        /// a DDX and a BLOB. First, a DDX (an XML representation of a DDS) is requested.
        /// The DDX is parsed and a DataDDS is created.
        /// The DDX contains a URL that points to the servers BLOB service. The BLOB
        /// service returns only the serialized binary content of the DataDDS. The DataDDS
        /// then deserializes the BLOB and fills itself with data.
        /// 
        /// </summary>
        /// <param name="statusUI">the <code>StatusUI</code> object to use for GUI updates
        /// and user cancellation notification (may be null).
        /// </param>
        /// <returns> The <code>DataDDS</code> object that results from applying the
        /// given CE, combined with this object's sticky CE, on the referenced
        /// dataset.
        /// </returns>
        /// <throws>  MalformedURLException if the URL given to the constructor </throws>
        /// <summary>                               has an error
        /// </summary>
        /// <throws>  IOException           if any error connecting to the remote server </throws>
        /// <throws>  ParseException        if the DDS parser returned an error </throws>
        /// <throws>  DDSException          on an error constructing the DDS </throws>
        /// <throws>  DAP2Exception         if any error returned by the remote server </throws>
        /// <opendap.ddx.experimental>  </opendap.ddx.experimental>
        public DataDDS GetDDXData(IStatusUI statusUI)
        {
            return GetDDXData("", statusUI, new DefaultFactory());
        }

        /// <summary> Returns the <code>ServerVersion</code> of the last connection.
        /// 
        /// </summary>
        /// <returns> the <code>ServerVersion</code> of the last connection.
        /// </returns>
        public ServerVersion GetServerVersion()
        {
            return ver;
        }

        /// <summary> A primitive parser for the MIME headers used by OPeNDAP.  This is used when
        /// reading from local sources of OPeNDAP Data objects. It is called by
        /// <code>ReadData</code> to simulate the important actions of the
        /// <code>URLConnection</code> MIME header parsing performed in
        /// <code>OpenConnection</code> for HTTP URL's.
        /// <p/>
        /// <b><i>NOTE:</b></i> Because BufferedReader seeks ahead, and therefore
        /// removescharacters from the InputStream which are needed later, and
        /// because there is no way to construct an InputStream from a
        /// BufferedReader, we have to use DataInputStream to Read the header
        /// lines, which triggers an unavoidable deprecated warning from the
        /// Java compiler.
        /// 
        /// </summary>
        /// <param name="is_Renamed">the InputStream to Read.
        /// </param>
        /// <returns> the InputStream to Read data from (after attaching any
        /// necessary decompression filters).
        /// </returns>
        /// <throws>  IOException   if any IO error. </throws>
        /// <throws>  DAP2Exception if the server returned an Error. </throws>
        private Stream parseMime(Stream is_Renamed)
        {
            // NOTE: because BufferedReader seeks ahead, and therefore removes
            // characters from the InputStream which are needed later, and
            // because there is no way to construct an InputStream from a
            // BufferedReader, we have to use DataInputStream to Read the header
            // lines, which triggers an unavoidable deprecated warning from the
            // Java compiler.

            //UPGRADE_TODO: Class 'java.io.DataInputStream' was converted to 'System.IO.BinaryReader' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javaioDataInputStream'"
            BinaryReader d = new BinaryReader(is_Renamed);

            String description = null;
            String encoding = null;

            // while there are more header (non-blank) lines
            String line;
            //UPGRADE_TODO: Method 'java.io.DataInputStream.readLine' was converted to 'System.IO.BinaryReader.ReadString' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javaioDataInputStreamreadLine'"
            while (!(line = d.ReadString()).Equals(""))
            {
                int spaceIndex = line.IndexOf(' ');
                // all header lines should have a space in them, but if not, skip ahead
                if (spaceIndex == - 1)
                    continue;
                String header = line.Substring(0, (spaceIndex) - (0));
                String value_Renamed = line.Substring(spaceIndex + 1);

                if (header.Equals("Server:"))
                {
                    ver = new ServerVersion(value_Renamed);
                }
                else if (header.Equals("Content-Description:"))
                {
                    description = value_Renamed;
                }
                else if (header.Equals("Content-Encoding:"))
                {
                    encoding = value_Renamed;
                }
            }
            HandleContentDesc(is_Renamed, description);
            return HandleContentEncoding(is_Renamed, encoding);
        }

        /// <summary> This code handles the Content-Description: header for
        /// <code>OpenConnection</code> and <code>parseMime</code>.
        /// Throws a <code>DAP2Exception</code> if the type is
        /// <code>dods_error</code>.
        /// 
        /// </summary>
        /// <param name="is_Renamed">  the InputStream to Read.
        /// </param>
        /// <param name="type">the Content-Description header, or null.
        /// </param>
        /// <throws>  IOException   if any error reading from the server. </throws>
        /// <throws>  DAP2Exception if the server returned an error. </throws>
        private static void HandleContentDesc(Stream is_Renamed, String type)
        {
            if (type != null && type.Equals("dods_error"))
            {
                // create server exception object
                DAP2Exception ds = new DAP2Exception();
                // parse the Error object from stream and throw it
                ds.parse(is_Renamed);
                throw ds;
            }
        }

        /// <summary> This code handles the Content-type: header for
        /// <code>OpenConnection</code> and <code>parseMime</code>
        /// 
        /// </summary>
        /// <param name="is_Renamed">      the InputStream to Read.
        /// </param>
        /// <param name="encoding">the Content-type header, or null.
        /// </param>
        /// <returns> the new InputStream, after applying an
        /// <code>InflaterInputStream</code> filter if necessary.
        /// </returns>
        private static Stream HandleContentEncoding(Stream is_Renamed, String encoding)
        {
            if (encoding != null && encoding.Equals("deflate"))
            {
                //UPGRADE_ISSUE: Constructor 'java.util.zip.InflaterInputStream.InflaterInputStream' was not converted. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1000_javautilzipInflaterInputStream'"
                return new InflaterInputStream(is_Renamed);
            }
            else
            {
                MemoryStream memStream = new MemoryStream(100);
                SupportClass.CopyStream(is_Renamed, memStream);
                memStream.Position = 0;
                return memStream;
            }
        }
    }
}